kmeans_model_new

  1# -*- coding: utf-8 -*-
  2"""KMeans_model.ipynb
  3
  4This script trains a KMeans model on student data, performs clustering, and finds matching students based on input data.
  5Automatically generated by Colab.
  6
  7Original file is located at
  8    https://colab.research.google.com/drive/1NdtQgxHskYoeVWsczWuXAYcstjXB9xLO
  9"""
 10
 11import json
 12import numpy as np
 13import pandas as pd
 14from sklearn.cluster import KMeans
 15from sklearn.preprocessing import MultiLabelBinarizer, StandardScaler
 16from sklearn.impute import SimpleImputer
 17from sklearn.metrics import pairwise_distances
 18import joblib  # For saving the model
 19
 20def load_json_data(file_path):
 21    """
 22    Load student data from a JSON file.
 23
 24    Args:
 25        file_path (str): Path to the JSON file containing student data.
 26
 27    Returns:
 28        list: List of student data as dictionaries.
 29    """
 30    with open(file_path, 'r') as file:
 31        data = json.load(file)
 32    return data
 33
 34# Load student data from the JSON file
 35file_path = 'students_large_dataset.json'  # Replace with your file path
 36students = load_json_data(file_path)
 37
 38# Convert JSON data to a pandas DataFrame
 39student_df = pd.DataFrame(students)
 40
 41# Standardize subject names to lowercase for uniformity
 42student_df['subjects'] = student_df['subjects'].apply(lambda x: [s.lower() for s in x])
 43
 44# Encode subjects using MultiLabelBinarizer
 45mlb = MultiLabelBinarizer()
 46subjects_encoded = mlb.fit_transform(student_df["subjects"])
 47
 48# Calculate average study hours for each student
 49avg_study_hours = student_df["study_hours"].apply(np.mean).values.reshape(-1, 1)
 50
 51# Encode goals using a predefined mapping (example mapping shown)
 52goal_mapping = {
 53    "Exam preparation": 0,
 54    "Assignments": 1,
 55    "Concept review": 2,
 56    "Project work": 3,
 57    "Homework": 4,
 58    "Practice": 5
 59}
 60goals_encoded = student_df["goals"].map(goal_mapping).values.reshape(-1, 1)
 61
 62# Combine encoded subjects, average study hours, and goals for each student
 63features = np.hstack([subjects_encoded, avg_study_hours, goals_encoded])
 64
 65# Impute missing values in the features using the mean of each column
 66imputer = SimpleImputer(strategy="mean")
 67features_imputed = imputer.fit_transform(features)
 68
 69# Scale the imputed features to have zero mean and unit variance
 70scaler = StandardScaler()
 71features_scaled = scaler.fit_transform(features_imputed)
 72
 73# Train the KMeans model to cluster students into 3 groups
 74kmeans = KMeans(n_clusters=3, random_state=42)
 75student_df["cluster"] = kmeans.fit_predict(features_scaled)
 76
 77# Save the trained KMeans model, scaler, imputer, and MultiLabelBinarizer for later use
 78joblib.dump(kmeans, 'kmeans_model.pkl')
 79joblib.dump(scaler, 'scaler.pkl')
 80joblib.dump(imputer, 'imputer.pkl')
 81joblib.dump(mlb, 'mlb.pkl')
 82
 83print("KMeans model, scaler, and MultiLabelBinarizer have been saved.")
 84
 85def find_matching_students(new_student, student_df, kmeans, scaler, mlb, n_matches=3):
 86    """
 87    Find the top matching students for a new student based on their study profile.
 88
 89    Args:
 90        new_student (dict): The new student's data (name, subjects, study hours, goals).
 91        student_df (pd.DataFrame): DataFrame containing the student data.
 92        kmeans (KMeans): Trained KMeans model to predict the cluster for the new student.
 93        scaler (StandardScaler): Pre-trained scaler to standardize the features.
 94        mlb (MultiLabelBinarizer): Pre-trained MultiLabelBinarizer to encode subjects.
 95        n_matches (int): The number of closest matching students to return (default is 3).
 96
 97    Returns:
 98        pd.DataFrame: The `n_matches` closest students from the same cluster.
 99    """
100    # Process new student data
101    new_subjects = mlb.transform([[s.lower() for s in new_student["subjects"]]])[0]
102    new_avg_hour = np.mean(new_student["study_hours"])
103    new_goal_encoded = goal_mapping[new_student["goals"]]
104
105    # Create a feature vector for the new student
106    new_student_features = np.hstack([new_subjects, new_avg_hour, new_goal_encoded]).reshape(1, -1)
107    new_student_features_scaled = scaler.transform(new_student_features)
108
109    # Predict the cluster for the new student
110    cluster = kmeans.predict(new_student_features_scaled)[0]
111
112    # Filter the students to those in the same cluster
113    cluster_students = student_df[student_df["cluster"] == cluster]
114    cluster_features = features_scaled[student_df["cluster"] == cluster]
115
116    # Calculate pairwise Euclidean distances to find the closest matches
117    distances = pairwise_distances(new_student_features_scaled, cluster_features, metric="euclidean").flatten()
118    closest_indices = distances.argsort()[:n_matches]
119
120    # Return the closest matching students
121    return cluster_students.iloc[closest_indices]
122
123# Append a new student and find matching students
124new_student = {
125    "name": "KULDEEP",
126    "subjects": ["math", "biology"],
127    "study_hours": [12, 17],
128    "goals": "Exam preparation"
129}
130
131# Load previously saved models and encoders
132kmeans_model = joblib.load('kmeans_model.pkl')
133scaler = joblib.load('scaler.pkl')
134mlb = joblib.load('mlb.pkl')
135
136# Find the top 5 matching students for the new student
137matching_students = find_matching_students(new_student, student_df, kmeans_model, scaler, mlb, n_matches=5)
138
139# Print the top matching students' details
140print("Top matching students:")
141print(matching_students[["name", "subjects", "study_hours", "goals"]])
142
143# Append the new student to the dataset and save it back to the JSON file
144students.append(new_student)
145with open(file_path, 'w') as file:
146    json.dump(students, file, indent=4)
147
148# Reload data and prepare features for clustering
149student_df = pd.DataFrame(students)
150student_df['subjects'] = student_df['subjects'].apply(lambda x: [s.lower() for s in x])
151subjects_encoded = mlb.fit_transform(student_df["subjects"])
152avg_study_hours = student_df["study_hours"].apply(np.mean).values.reshape(-1, 1)
153goals_encoded = student_df["goals"].map(goal_mapping).values.reshape(-1, 1)
154
155# Impute and scale features
156features = np.hstack([subjects_encoded, avg_study_hours, goals_encoded])
157imputer = SimpleImputer(strategy="mean")
158features_imputed = imputer.fit_transform(features)  # Impute NaNs
159
160scaler = StandardScaler()
161features_scaled = scaler.fit_transform(features_imputed)  # Scale the imputed features
162
163# Train and save the updated KMeans model
164kmeans = KMeans(n_clusters=3, random_state=42)
165student_df["cluster"] = kmeans.fit_predict(features_scaled)
166joblib.dump(kmeans, 'kmeans_model.pkl')
167joblib.dump(scaler, 'scaler.pkl')
168joblib.dump(imputer, 'imputer.pkl')
169joblib.dump(mlb, 'mlb.pkl')
170
171print("Retrained model and updated data have been saved.")